
#读取csv文件part-00001-ada4b76e-d96c-4cb5-bb12-a60b47a276c8-c000.csv，分隔符为’\\x7f\\x5e’,engin=‘python’ ，文件格式调整为parquet
import pandas as pd
df = pd.read_csv('part-00001-ada4b76e-d96c-4cb5-bb12-a60b47a276c8-c000.csv',sep='\\x7f\\x5e',engine='python')
print(df.head(1))

# 定义要分析的字段列表
score_fields = ['SCORE_557', 'QYZXMODEL', 'HNDGMODEL', 'JXJKMODEL', 'FICOMODEL', 'GRZXMODEL', 'HNGRMODEL', 'GSMODEL']


# 确保 APPLY_DT 的格式正确
print(df.head(1))
df['APPLY_DT'] = pd.to_datetime(df['APPLY_DT'])


# 定义Y标签
df['Y'] = 2  # 默认值为2
df.loc[df['PFLAG_30D'] >= 30, 'Y'] = 1  # 逾期30天及以上为1
df.loc[df['PFLAG_30D'] <= 0, 'Y'] = 0   # 未逾期为0

# 剔除Y=2的样本
df = df[df['Y'] != 2]

# 导入必要的库
import numpy as np
from sklearn.metrics import roc_curve, auc
import matplotlib.pyplot as plt

print(df.columns)
# 对每个评分字段计算KS/AUC/GINI
for field in score_fields:
    valid_data = df[[field, 'Y', 'APPLY_DT']].dropna()
    
    if len(valid_data) == 0:
        print(f'\n{field}字段数据全部为空，跳过计算')
        continue
    
    # 按日期拆分数据集
    cutoff_date = pd.to_datetime('2024-05-31')
    
    # 分离训练验证集和OOT集
    train_val_data = valid_data[valid_data['APPLY_DT'] <= cutoff_date]
    oot_data = valid_data[valid_data['APPLY_DT'] > cutoff_date]
    
    # 将训练验证集按6:4比例随机分割
    from sklearn.model_selection import train_test_split
    train_data, val_data = train_test_split(train_val_data, train_size=0.6, random_state=42)
    
    # 分别计算训练集、验证集和OOT集的指标
    for dataset_name, dataset in [('训练集', train_data), ('验证集', val_data), ('OOT集', oot_data)]:
        if len(dataset) == 0:
            print(f'\n{field}的{dataset_name}为空，跳过计算')
            continue
            
        # 计算ROC曲线
        fpr, tpr, thresholds = roc_curve(dataset['Y'], dataset[field], pos_label=1)
        
        # 计算KS值
        ks = max(abs(tpr - fpr))
        
        # 计算AUC值
        auc_value = auc(fpr, tpr)
        
        # 计算GINI系数
        gini = 2 * auc_value - 1
        
        # 绘制ROC曲线
        plt.figure(figsize=(8, 6))
        plt.plot(fpr, tpr, label=f'ROC curve (AUC = {auc_value:.3f})')
        plt.plot([0, 1], [0, 1], 'k--')
        plt.xlim([0.0, 1.0])
        plt.ylim([0.0, 1.05])
        plt.xlabel('False Positive Rate')
        plt.ylabel('True Positive Rate')
        plt.title(f'{field} - {dataset_name} ROC Curve\nKS={ks:.3f}, GINI={gini:.3f}')
        plt.legend(loc="lower right")
        plt.grid(True)
        plt.show()
        
        print(f'\n{field}的{dataset_name}评估指标:')
        print(f'样本量: {len(dataset)}')
        print(f'KS值: {ks:.3f}')
        print(f'AUC值: {auc_value:.3f}')
        print(f'GINI系数: {gini:.3f}')
